﻿using Snowflake.Core;
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text; 
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;


namespace homework9
{
   public  class Crawler
    {
        private readonly static IdWorker worker = new IdWorker(1, 1);
        public static readonly string urlRegex = @"(href|HREF)[]*=[]*[""'][^""'#>]+(.html|.htm|.aspx|.jsp)[""']";//在HTML文本中查找URL
        public Hashtable pageInfo = new Hashtable();
        public Queue<string> waiting = new Queue<string>();
        public Encoding Encoding { get; set; }
        public Hashtable Urls { get; set; }
        public Crawler()
        {
            Encoding = Encoding.UTF8;
        }
        public void Crawl() { }
        public string DownLoad(string url)
        {
            WebClient webClient = new WebClient
            {
                Encoding = Encoding.UTF8
            };
            string html = webClient.DownloadString(url);
            string fileName = "";
                if (Regex.IsMatch(url, @".*.html?$"))
                {
                    fileName = worker.NextId().ToString() + ".html";
                }
                else if (Regex.IsMatch(url, @".*.jsp?$"))
                {
                    fileName = worker.NextId().ToString() + ".jsp";
                }
                else if (Regex.IsMatch(url, @".*.aspx?$"))
                {
                    fileName = worker.NextId().ToString() + ".aspx";
                }
                else
                {
                    fileName = worker.NextId().ToString();
                }
                if (!Directory.Exists("E://data"))
                {
                    Directory.CreateDirectory("E://data");
                }
                File.WriteAllText("E://data//" + fileName, html, Encoding.UTF8);
                return html;
        }
        public void Parse(string html, string pageUrl)
        {
            MatchCollection matchUrls = new Regex(urlRegex).Matches(html);
            foreach (Match matchUrl in matchUrls)
            {
                string linkUrl = matchUrl.Value.Substring(matchUrl.Value.IndexOf('=') + 1)
                          .Trim('"', '\"', '#', '>');
                if (linkUrl == null || linkUrl.Equals(""))
                {
                    continue;
                }
                //linkUrl转换为绝对路径   pageUrl当前页url
                linkUrl = Change(linkUrl, pageUrl);
                if (!Urls.ContainsKey(linkUrl))
                {
                    waiting.Enqueue(linkUrl);
                    Urls.Add(linkUrl, false);//加入页面
                }
            }
        }
        private string Change(string url, string baseUrl)
        {
            if (url.Contains("://"))
            {
                return url;
            }
            if (url.StartsWith("//"))
            {
                return "http:" + url;
            }
            if (url.StartsWith("/"))
            {
                String temp = Regex.Match(baseUrl, urlRegex).Value;
                return temp.EndsWith("/") ? temp + url.Substring(1) : temp + url;
            }
            if (url.StartsWith("./"))
            {
                return Change(url.Substring(2), baseUrl);
            }
            if (url.StartsWith("../"))
            {
                int idx = baseUrl.LastIndexOf('/');
                return Change(url.Substring(3), baseUrl.Substring(0, idx));
            }
            int end = baseUrl.LastIndexOf("/");
            return baseUrl.Substring(0, end) + "/" + url;
        }
    }
}